## In order to replicate the results shown in Figure 5 and in additional figures, showcasing the case without weight decay
## for image classification experiments, please use the following .py files and associated params.

Files are divided among folders based on which dataset they were using, so, fro example, to
replicate CIFAR10 experiment main_ada_lr_d.py in CIFAR10 folder should be used. 

By default seed = 1 is chosen; in order to change it additional argument --seed needs to be passed.

CIFAR10 and CIFAR 100 data paths can be modified in utils.py file.

In the case of using the code on a machine with multiple GPU's, call to main should be preceeded with
export CUDA_VISIBLE_DEVICES=0;

main_scale.py realizes ''Fixed LR'' OASIS variant
main_scale_momentum.py realizes ''Momentum'' OASIS variant
main_ada_lr_d.py realizes fully adaptive OASIS variant

#----------------------------------------------------------------------
# ResNet20 on CIFAR 10 - With Weight Decay
#----------------------------------------------------------------------

SGD:
main_ada_lr_d.py --optimizer sgd --lr 0.1 --weight-decay 0.0005 --depth 20

ADAM:
main_ada_lr_d.py --optimizer adam --lr 0.001 --weight-decay 0.0005 --depth 20

ADAMW:
main_ada_lr_d.py --optimizer adamw --lr 0.01 --weight-decay 0.0005 --depth 20

AdaHessian:
main_ada_lr_d.py --optimizer adahessian --lr 0.15 --weight-decay 0.0005 --depth 20

OASIS - Adaptive LR:
main_ada_lr_d.py --optimizer adaadahessian --alpha 0.1 --beta 0.98 --zeta 1.0 --gamma 0.1 --rho 0.1 --weight-decay 0.001 --depth 20 --warmstart-samples-idbatch 500

OASIS - Fixed LR:
main_scale.py --optimizer adaadahessian --lr 0.1 --alpha 0.1 --beta 0.95 --weight-decay 0.0005 --depth 20

OASIS - Momentum:
main_scale_momentum.py --optimizer adaadahessian --lr 0.1 --alpha 0.1 --beta 0.99 --weight-decay 0.0005 --depth 20

#----------------------------------------------------------------------
# ResNet20 on CIFAR 10 - Without Weight Decay
#----------------------------------------------------------------------

SGD:
main_ada_lr_d.py --optimizer sgd --lr 0.15 --weight-decay 0.0 --depth 20

ADAM:
main_ada_lr_d.py --optimizer adam --lr 0.005 --weight-decay 0.0 --depth 20

ADAMW:
main_ada_lr_d.py --optimizer adamw --lr 0.005 --weight-decay 0.0 --depth 20

AdaHessian:
main_ada_lr_d.py --optimizer adahessian --lr 0.25 --weight-decay 0.0 --depth 20

OASIS - Adaptive LR:
main_ada_lr_d.py --optimizer adaadahessian --alpha 0.1 --beta 0.999 --zeta 1.0 --gamma 0.01 --rho 0.1 --weight-decay 0.0 --depth 20 --warmstart-samples-idbatch 500

OASIS - Fixed LR:
main_scale.py --optimizer adaadahessian --lr 0.1 --alpha 0.01 --beta 0.999 --weight-decay 0.0 --depth 20

OASIS - Momentum:
main_scale_momentum.py --optimizer adaadahessian --lr 0.1 --alpha 0.01 --beta 0.999 --weight-decay 0.0 --depth 20

#----------------------------------------------------------------------
# ResNet32 on CIFAR 10 - With Weight Decay
#----------------------------------------------------------------------

SGD:
main_ada_lr_d.py --optimizer sgd --lr 0.1 --weight-decay 0.0005 --depth 32

ADAM:
main_ada_lr_d.py --optimizer adam --lr 0.001 --weight-decay 0.0005 --depth 32

ADAMW:
main_ada_lr_d.py --optimizer adamw --lr 0.01 --weight-decay 0.0005 --depth 32

AdaHessian:
main_ada_lr_d.py --optimizer adahessian --lr 0.15 --weight-decay 0.0005 --depth 32

OASIS - Adaptive LR:
main_ada_lr_d.py --optimizer adaadahessian --alpha 0.1 --beta 0.98 --zeta 1.0 --gamma 0.1 --rho 0.5 --weight-decay 0.001 --depth 32 --warmstart-samples-idbatch 500

OASIS - Fixed LR:
main_scale.py --optimizer adaadahessian --lr 0.125 --alpha 0.1 --beta 0.98 --weight-decay 0.0005 --depth 32

OASIS - Momentum:
main_scale_momentum.py --optimizer adaadahessian --lr 1.0 --alpha 0.1 --beta 0.98 --weight-decay 0.0005 --depth 32

#----------------------------------------------------------------------
# ResNet32 on CIFAR 10 - Without Weight Decay
#----------------------------------------------------------------------

SGD:
main_ada_lr_d.py --optimizer sgd --lr 0.1 --weight-decay 0.0005 --depth 32

ADAM:
main_ada_lr_d.py --optimizer adam --lr 0.001 --weight-decay 0.0005 --depth 32

ADAMW:
main_ada_lr_d.py --optimizer adamw --lr 0.01 --weight-decay 0.0005 --depth 32

AdaHessian:
main_ada_lr_d.py --optimizer adahessian --lr 0.15 --weight-decay 0.0005 --depth 32

OASIS - Adaptive LR:
main_ada_lr_d.py --optimizer adaadahessian --alpha 0.1 --beta 0.999 --zeta 1.0 --gamma 0.01 --rho 0.1 --weight-decay 0.001 --depth 32 --warmstart-samples-idbatch 500

OASIS - Fixed LR:
main_scale.py --optimizer adaadahessian --lr 0.5 --alpha 0.1 --beta 0.999 --weight-decay 0.0005 --depth 32

OASIS - Momentum:
main_scale_momentum.py --optimizer adaadahessian --lr 0.5 --alpha 0.1 --beta 0.999 --weight-decay 0.0005 --depth 32

#----------------------------------------------------------------------
# ResNet18 on CIFAR 100 - With Weight Decay
#----------------------------------------------------------------------

SGD:
main_ada_lr_d.py --optimizer sgd --lr 0.1 --weight-decay 0.0005

ADAM:
main_ada_lr_d.py --optimizer adam --lr 0.001 --weight-decay 0.0005

ADAMW:
main_ada_lr_d.py --optimizer adamw --lr 0.01 --weight-decay 0.0005

AdaHessian:
main_ada_lr_d.py --optimizer adahessian --lr 0.15 --weight-decay 0.0005

OASIS - Adaptive LR:
main_ada_lr_d.py --optimizer adaadahessian --alpha 0.1 --beta 0.98 --zeta 1.0 --gamma 0.1 --rho 0.2 --weight-decay 0.001 --warmstart-samples-idbatch 500

OASIS - Fixed LR:
main_scale.py --optimizer adaadahessian --lr 0.1 --alpha 0.1 --beta 0.98 --weight-decay 0.0005

OASIS - Momentum:
main_scale_momentum.py --optimizer adaadahessian --lr 0.1 --alpha 0.1 --beta 0.98 --weight-decay 0.0005

#----------------------------------------------------------------------
# ResNet18 on CIFAR 100 - Without Weight Decay
#----------------------------------------------------------------------

SGD:
main_ada_lr_d.py --optimizer sgd --lr 0.15 --weight-decay 0.0

ADAM:
main_ada_lr_d.py --optimizer adam --lr 0.005 --weight-decay 0.0

ADAMW:
main_ada_lr_d.py --optimizer adamw --lr 0.005 --weight-decay 0.0

AdaHessian:
main_ada_lr_d.py --optimizer adahessian --lr 0.25 --weight-decay 0.0

OASIS - Adaptive LR:
main_ada_lr_d.py --optimizer adaadahessian --alpha 0.1 --beta 0.999 --zeta 1.0 --gamma 0.01 --rho 0.2 --weight-decay 0.0 --warmstart-samples-idbatch 500

OASIS - Fixed LR:
main_scale.py --optimizer adaadahessian --lr 0.1 --alpha 0.01 --beta 0.999 --weight-decay 0.0

OASIS - Momentum:
main_scale_momentum.py --optimizer adaadahessian --lr 0.1 --alpha 0.01 --beta 0.999 --weight-decay 0.0